#-*-coding:utf-8-*-
import scrapy
from CrawlJob.items import CrawljobItem		
import json
from scrapy import Request
from scrapy.contrib.spiders import CrawlSpider
import re
from bs4 import BeautifulSoup
import sys
reload(sys)
sys.setdefaultencoding("utf-8")
class MySpider(CrawlSpider):
	name="job5"
	allowed_domains = ['search.51job.com','jobs.51job.com']#'软件工程师','硬件工程师','数据挖掘','C++','java','计算机','数据分析''测试工程师','需求分析','系统架构','运营','前端','ui','测试工程师','软件开发',
	start_urls = ['http://search.51job.com']
	jobs=['需求分析','软件开发','系统架构']
	headers = {'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'}
	def start_requests(self):
		for pro in self.jobs:
			u = "http://search.51job.com/jobsearch/search_result.php?keyword=" + pro.encode('utf-8').decode('utf-8')
  			yield scrapy.Request(u, callback=self.parse)
	def parse(self, response):
		for sel in response.css("html body div.dw_wp div#resultList.dw_table div.el p.t1 span a"): 
			url = sel.re('href="(.*?)"')[0]
			yield scrapy.Request(url,callback=self.parse_item)
		nextpage=response.xpath('/html/body/div[2]/div[6]/div/div/div/ul/li[8]/a/@href').extract()[0]
		if nextpage.strip():
			print nextpage
			yield scrapy.Request(nextpage,callback=self.parse)	
	def parse_item(self, response):
		item = CrawljobItem()
		item['minoffer']=self.Getminoffer(response)
		item['maxoffer']=self.Getmaxoffer(response)
		item['city']=self.Getcity(response)
		item['title']=self.Gettitle(response)
		item['company'] =self.Getcompany(response)
		item['location']=self.Getlocation(response)
		item['education']=self.Geteducation(response)
		item['work_experience']=self.Getwork_experience(response)
		item['trades']=self.Gettrades(response)
		item['companyType']=self.GetcompanyType(response)
		item['companySize']=self.GetcompanySize(response)
		item['url']=self.Geturl(response)
		yield item
	# def Getminoffer(self, response):
	# 	Minoffer=0.0
	# 	offer = response.xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/strong/text()').extract()[0].encode('utf-8').decode('utf-8')
	# 	if offer.strip():
	# 		s=re.findall(u'(.*?)-', offer)
	# 		if '万/年' in offer:
	# 			Minoffer=float(s[0]) * 10000 / 12
	# 		elif '万/月' in offer:
	# 			Minoffer=float(s[0]) * 10000
	# 		else:
	# 			Minoffer=float(s[0]) * 1000
	# 	else:
	# 		Minoffer=10000
	# 	return Minoffer
	# def Getmaxoffer(self, response):
	# 	Maxoffer=0.0
	# 	offer = response.xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/strong/text()').extract()[0].encode('utf-8').decode('utf-8')
	# 	if offer.strip():
	# 		if '万/年' in offer:
	# 			s = re.findall(u'-(.*?)万', offer)
	# 			Maxoffer=float(s[0]) * 10000 / 12
	# 		elif '万/月' in offer:
	# 			s=re.findall(u'-(.*?)万',offer)
	# 			Maxoffer=float(s[0]) * 10000
	# 		else:
	# 			s=re.findall(u'-(.*?)千',offer)
	# 			Maxoffer=float(s[0]) * 1000
	# 	else:
	# 		Maxoffer=30000
	# 	return Maxoffer
	# def Getcity(self, response):
	# 	city = response.xpath('//span[@class="lname"]/text()').extract()[0].encode('utf-8').decode('utf-8')
	# 	if city=='':
	# 		city = '北京'
	# 	return city
	# def Geturl(self, response):
	# 	url = response.url
	# 	return url
	# def Getcompany(self, response):
	# 	company=response.xpath('//p[@class="cname"]/a/@title').extract()[0].encode('utf-8').decode('utf-8')
	# 	if company=='':
	# 		company = "error"
	# 	return company
	# def GetcompanyType(self, response):
	# 	trades = response.xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/p[2]/text()').extract()[0].replace(u'\xa0', u'').replace(" ",'').encode('utf-8').decode('utf-8')
	# 	trades = ' '.join(trades.split())
	# 	if trades == '':
	# 		companyType='民营公司'.encode('utf-8').decode('utf-8')
	# 		return companyType
	# 		pass
	# 	companyType = re.split(' |', trades)[0].encode('utf-8').decode('utf-8')
	# 	if companyType.strip() == 0:
	# 		companyType='民营公司'.encode('utf-8').decode('utf-8')
	# 	if '公司' in companyType: 
	# 		return companyType
	# 	else:
	# 		companyType='民营公司'.encode('utf-8').decode('utf-8')
	# 		return companyType
	# def GetcompanySize(self, response):
	# 	trades = response.xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/p[2]/text()').extract()[0].replace(u'\xa0', u'').replace(" ",'').encode('utf-8').decode('utf-8')
	# 	trades = ' '.join(trades.split())
	# 	if trades == '':
	# 		companySize='50-150人'.encode('utf-8').decode('utf-8')
	# 		return companySize
	# 		pass
	# 	companySize = re.split(' |', trades)[1].encode('utf-8').decode('utf-8').replace('|', '')
	# 	if companySize.strip() == 0:
	# 		companySize='50-100人'.encode('utf-8').decode('utf-8')
	# 	if '人' in companySize:
	# 		return companySize
	# 	else:
	# 		companySize='50-100人'.encode('utf-8').decode('utf-8')
	# 		return companySize
	# def Gettrades(self, response):
	# 	trades = response.xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/p[2]/text()').extract()[0].replace(u'\xa0', u'').replace(" ",'').encode('utf-8').decode('utf-8')
	# 	trades = ' '.join(trades.split())
	# 	if trades=='':
	# 		trades='计算机软件'.encode('utf-8').decode('utf-8')
	# 		return trades
	# 		pass
	# 	for i in xrange(1, 3):
	# 		trades = re.split(' |', trades)[i].encode('utf-8').decode('utf-8').replace('|', '')
	# 		if '/' in trades:
	# 			return trades
	# 		else:
	# 			trades='计算机软件'.encode('utf-8').decode('utf-8')
	# 			return trades
	# def Getlocation(self, response):
	# 	location=response.xpath('//p[@class="fp"]/text()').extract()[1].rstrip()
	# 	if location=='':
	# 		location=response.xpath('//span[@class="lname"]/text()').extract()[0].encode('utf-8').decode('utf-8')
	# 		if location=='':
	# 			location='北京'
	# 			return location
	# 		return location
	# 	return location
	# def Gettitle(self, response):
	# 	title=response.xpath('//div[@class="cn"]/h1/@title').extract()[0].encode('utf-8').decode('utf-8')
	# 	return title
	# def Geteducation(self, response):
	# 	soup = BeautifulSoup(response.body)
	# 	ss = soup.find_all('span',class_='sp4')
	# 	education=''
	# 	for s in ss:
	# 		edu=s.get_text().encode('utf-8').decode('utf-8')
	# 		if '专' in edu:
	# 			education = edu.encode('utf-8').decode('utf-8')
	# 			break
	# 		elif '本' in edu:
	# 			education = edu.encode('utf-8').decode('utf-8')
	# 			break
	# 		else:
	# 			education='暂无明确要求'.encode('utf-8').decode('utf-8')
	# 	return education
	# def Getwork_experience(self, response):
	# 	soup = BeautifulSoup(response.body)
	# 	ss = soup.find_all('span',class_='sp4')
	# 	education=''
	# 	for s in ss:
	# 		edu=s.get_text().encode('utf-8').decode('utf-8')
	# 		if '经验' in edu:
	# 			education = edu.encode('utf-8').decode('utf-8')
	# 			break
	# 		else:
	# 			education='暂无要求'.encode('utf-8').decode('utf-8')
	# 	return education
	def Getminoffer(self, response):
		Minoffer=0.0
		try:
			offer = response.xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/strong/text()').extract()[0].encode('utf-8').decode('utf-8')
			if offer.strip():
				s=re.findall(u'(.*?)-', offer)
				if '万/年' in offer:
					Minoffer=float(s[0]) * 10000 / 12
				elif '万/月' in offer:
					Minoffer=float(s[0]) * 10000
				else:
					Minoffer=float(s[0]) * 1000
			else:
				Minoffer=10000
		except:
			Minoffer=10000
		return Minoffer
	def Getmaxoffer(self, response):
		Maxoffer=0.0
		try:
			offer = response.xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/strong/text()').extract()[0].encode('utf-8').decode('utf-8')
			if offer.strip():
				if '万/年' in offer:
					s = re.findall(u'-(.*?)万', offer)
					Maxoffer=float(s[0]) * 10000 / 12
				elif '万/月' in offer:
					s=re.findall(u'-(.*?)万',offer)
					Maxoffer=float(s[0]) * 10000
				else:
					s=re.findall(u'-(.*?)千',offer)
					Maxoffer=float(s[0]) * 1000
			else:
				Maxoffer=20000
		except:
			Maxoffer=20000
		return Maxoffer
	def Getcity(self, response):
		try:
			city = response.xpath('//span[@class="lname"]/text()').extract()[0].encode('utf-8').decode('utf-8')
			ci=''
			if '-'in city:
				ci=re.findall('(.*?)-',city)
			for c in ci:
				city=''.join(c.encode('utf-8').decode('utf-8'))
			if city=='':
				city = '北京'
		except:
			city='北京'
		return city
	def Geturl(self, response):
		url = response.url
		return url
	def Getcompany(self, response):
		try:
			company=response.xpath('//p[@class="cname"]/a/@title').extract()[0].encode('utf-8').decode('utf-8')
			if company=='':
				company = "None"
		except:
			company='None'
		return company
	def GetcompanyType(self, response):
		try:
			trades = response.xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/p[2]/text()').extract()[0].replace(u'\xa0', u'').replace(" ",'').encode('utf-8').decode('utf-8')
			trades = ' '.join(trades.split())
			companyType = re.split(' |', trades)[0].encode('utf-8').decode('utf-8')
			if '公司' in companyType: 
				pass
			elif '国企' in companyType:
				companyType='国有企业'.encode('utf-8').decode('utf-8')
			elif '外资' in companyType:
				pass
			else:
				companyType='民营公司'.encode('utf-8').decode('utf-8')
		except:
			companyType='民营公司'
		return companyType
	def GetcompanySize(self, response):
		trades = response.xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/p[2]/text()').extract()[0].replace(u'\xa0', u'').replace(" ",'').encode('utf-8').decode('utf-8')
		trades = ' '.join(trades.split())
		if trades == '':
			companySize='50-150人'.encode('utf-8').decode('utf-8')
			return companySize
		try:
			companySize = re.split(' |', trades)[1].encode('utf-8').decode('utf-8').replace('|', '')
			if companySize.strip() == 0:
				companySize='50-100人'.encode('utf-8').decode('utf-8')
			if '人' in companySize:
				return companySize
			else:
				companySize='50-100人'.encode('utf-8').decode('utf-8')
				return companySize
		except:
			companySize='50-100人'.encode('utf-8').decode('utf-8')
		return companySize
	def Gettrades(self, response):
		trades = response.xpath('/html/body/div[2]/div[2]/div[2]/div/div[1]/p[2]/text()').extract()[0].replace(u'\xa0', u'').replace(" ",'').encode('utf-8').decode('utf-8')
		trades = ' '.join(trades.split())
		if trades=='':
			trades='计算机软件'.encode('utf-8').decode('utf-8')
			return trades
		# for i in xrange(1, 3):
		try:
			trades = re.split(' |', trades)[2].encode('utf-8').decode('utf-8').replace('|', '')
			if '/' in trades:
					return trades
			else:
				trades='计算机软件'.encode('utf-8').decode('utf-8')
				return trades
			# trades = ''.join(trades.split())
		except:
			trades='计算机软件'.encode('utf-8').decode('utf-8')
		return trades
	def Getlocation(self, response):
		try:
			location=response.xpath('//p[@class="fp"]/text()').extract()[1].rstrip()
			if location=='':
				location=response.xpath('//span[@class="lname"]/text()').extract()[0].encode('utf-8').decode('utf-8')
				if location=='':
					location='北京'
		except:
			location='北京'
		return location
	def Gettitle(self, response):
		try:
			title=response.xpath('//div[@class="cn"]/h1/@title').extract()[0].encode('utf-8').decode('utf-8')
		except:
			title="None"
		return title
	def Geteducation(self, response):
		try:
			soup = BeautifulSoup(response.body)
			ss = soup.find_all('span',class_='sp4')
			education=''
			for s in ss:
				edu=s.get_text().encode('utf-8').decode('utf-8')
				if '专' in edu:
					education = edu.encode('utf-8').decode('utf-8')
					break
				elif '本' in edu:
					education = edu.encode('utf-8').decode('utf-8')
					break
				else:
					education='暂无明确要求'.encode('utf-8').decode('utf-8')
		except:
			education='暂无明确要求'.encode('utf-8').decode('utf-8')
		return education
	def Getwork_experience(self, response):
		try:
			soup = BeautifulSoup(response.body)
			ss = soup.find_all('span',class_='sp4')
			education=''
			for s in ss:
				edu=s.get_text().encode('utf-8').decode('utf-8')
				if '经验' in edu:
					education = edu.encode('utf-8').decode('utf-8')
					break
				else:
					education='暂无要求'.encode('utf-8').decode('utf-8')
		except:
			education='暂无要求'.encode('utf-8').decode('utf-8')
		return education
